1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.solr.servlet;
19
20 import static org.easymock.EasyMock.anyObject;
21 import static org.easymock.EasyMock.createMock;
22 import static org.easymock.EasyMock.expect;
23 import static org.easymock.EasyMock.replay;
24
25 import java.io.BufferedInputStream;
26 import java.io.ByteArrayInputStream;
27 import java.io.File;
28 import java.io.IOException;
29 import java.io.InputStream;
30 import java.lang.invoke.MethodHandles;
31 import java.net.URL;
32 import java.nio.charset.StandardCharsets;
33 import java.util.ArrayList;
34 import java.util.Collections;
35 import java.util.HashMap;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Vector;
39
40 import javax.servlet.ReadListener;
41 import javax.servlet.ServletInputStream;
42 import javax.servlet.http.HttpServletRequest;
43
44 import org.apache.commons.io.FileUtils;
45 import org.apache.commons.io.IOUtils;
46 import org.apache.solr.SolrTestCaseJ4;
47 import org.apache.solr.common.SolrException;
48 import org.apache.solr.common.params.CommonParams;
49 import org.apache.solr.common.params.MultiMapSolrParams;
50 import org.apache.solr.common.params.SolrParams;
51 import org.apache.solr.common.util.ContentStream;
52 import org.apache.solr.core.SolrCore;
53 import org.apache.solr.request.SolrQueryRequest;
54 import org.apache.solr.servlet.SolrRequestParsers.MultipartRequestParser;
55 import org.apache.solr.servlet.SolrRequestParsers.FormDataRequestParser;
56 import org.apache.solr.servlet.SolrRequestParsers.RawRequestParser;
57 import org.apache.solr.servlet.SolrRequestParsers.StandardRequestParser;
58 import org.junit.AfterClass;
59 import org.junit.BeforeClass;
60 import org.junit.Test;
61 import org.slf4j.Logger;
62 import org.slf4j.LoggerFactory;
63
64 public class SolrRequestParserTest extends SolrTestCaseJ4 {
65
66 private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
67
68 @BeforeClass
69 public static void beforeClass() throws Exception {
70 initCore("solrconfig.xml", "schema.xml");
71 parser = new SolrRequestParsers( h.getCore().getSolrConfig() );
72 }
73
74 static SolrRequestParsers parser;
75
76 @AfterClass
77 public static void afterClass() {
78 parser = null;
79 }
80
81 @Test
82 public void testStreamBody() throws Exception
83 {
84 String body1 = "AMANAPLANPANAMA";
85 String body2 = "qwertasdfgzxcvb";
86 String body3 = "1234567890";
87
88 SolrCore core = h.getCore();
89
90 Map<String,String[]> args = new HashMap<>();
91 args.put( CommonParams.STREAM_BODY, new String[] {body1} );
92
93
94 List<ContentStream> streams = new ArrayList<>();
95 SolrQueryRequest req = parser.buildRequestFrom( core, new MultiMapSolrParams( args ), streams );
96 assertEquals( 1, streams.size() );
97 assertEquals( body1, IOUtils.toString( streams.get(0).getReader() ) );
98 req.close();
99
100
101 streams = new ArrayList<>();
102 args.put( CommonParams.STREAM_BODY, new String[] {body1,body2,body3} );
103 req = parser.buildRequestFrom( core, new MultiMapSolrParams( args ), streams );
104 assertEquals( 3, streams.size() );
105 ArrayList<String> input = new ArrayList<>();
106 ArrayList<String> output = new ArrayList<>();
107 input.add( body1 );
108 input.add( body2 );
109 input.add( body3 );
110 output.add( IOUtils.toString( streams.get(0).getReader() ) );
111 output.add( IOUtils.toString( streams.get(1).getReader() ) );
112 output.add( IOUtils.toString( streams.get(2).getReader() ) );
113
114 Collections.sort( input );
115 Collections.sort( output );
116 assertEquals( input.toString(), output.toString() );
117 req.close();
118
119
120 String ctype = "text/xxx";
121 streams = new ArrayList<>();
122 args.put( CommonParams.STREAM_CONTENTTYPE, new String[] {ctype} );
123 req = parser.buildRequestFrom( core, new MultiMapSolrParams( args ), streams );
124 for( ContentStream s : streams ) {
125 assertEquals( ctype, s.getContentType() );
126 }
127 req.close();
128 }
129
130 @Test
131 public void testStreamURL() throws Exception
132 {
133 URL url = getClass().getResource("/README");
134 assertNotNull("Missing file 'README' in test-resources root folder.", url);
135
136 byte[] bytes = IOUtils.toByteArray(url);
137
138 SolrCore core = h.getCore();
139
140 Map<String,String[]> args = new HashMap<>();
141 args.put( CommonParams.STREAM_URL, new String[] { url.toExternalForm() } );
142
143
144 List<ContentStream> streams = new ArrayList<>();
145 try (SolrQueryRequest req = parser.buildRequestFrom( core, new MultiMapSolrParams( args ), streams )) {
146 assertEquals( 1, streams.size() );
147 try (InputStream in = streams.get(0).getStream()) {
148 assertArrayEquals( bytes, IOUtils.toByteArray( in ) );
149 }
150 }
151 }
152
153 @Test
154 public void testStreamFile() throws Exception
155 {
156 File file = getFile("README");
157
158 byte[] bytes = FileUtils.readFileToByteArray(file);
159
160 SolrCore core = h.getCore();
161
162 Map<String,String[]> args = new HashMap<>();
163 args.put( CommonParams.STREAM_FILE, new String[] { file.getAbsolutePath() } );
164
165
166 List<ContentStream> streams = new ArrayList<>();
167 try (SolrQueryRequest req = parser.buildRequestFrom( core, new MultiMapSolrParams( args ), streams )) {
168 assertEquals( 1, streams.size() );
169 try (InputStream in = streams.get(0).getStream()) {
170 assertArrayEquals( bytes, IOUtils.toByteArray( in ) );
171 }
172 }
173 }
174
175 @Test
176 public void testUrlParamParsing() throws Exception
177 {
178 final String[][] teststr = new String[][] {
179 { "this is simple", "this%20is%20simple" },
180 { "this is simple", "this+is+simple" },
181 { "\u00FC", "%C3%BC" },
182 { "\u0026", "%26" },
183 { "", "" },
184 { "\u20AC", "%E2%82%ac" }
185 };
186
187 for( String[] tst : teststr ) {
188 SolrParams params = SolrRequestParsers.parseQueryString( "val="+tst[1] );
189 assertEquals( tst[0], params.get( "val" ) );
190 params = SolrRequestParsers.parseQueryString( "val="+tst[1]+"&" );
191 assertEquals( tst[0], params.get( "val" ) );
192 params = SolrRequestParsers.parseQueryString( "&&val="+tst[1]+"&" );
193 assertEquals( tst[0], params.get( "val" ) );
194 params = SolrRequestParsers.parseQueryString( "&&val="+tst[1]+"&&&val="+tst[1]+"&" );
195 assertArrayEquals(new String[]{tst[0],tst[0]}, params.getParams("val") );
196 }
197
198 SolrParams params = SolrRequestParsers.parseQueryString("val");
199 assertEquals("", params.get("val"));
200
201 params = SolrRequestParsers.parseQueryString("val&foo=bar=bar&muh&");
202 assertEquals("", params.get("val"));
203 assertEquals("bar=bar", params.get("foo"));
204 assertEquals("", params.get("muh"));
205
206 final String[] invalid = {
207 "q=h%FCllo",
208 "q=h\u00FCllo",
209 "q=hallo%",
210 "q=hallo%1",
211 "q=hallo%XX123",
212 "=hallo"
213 };
214 for (String s : invalid) {
215 try {
216 SolrRequestParsers.parseQueryString(s);
217 fail("Should throw SolrException");
218 } catch (SolrException se) {
219
220 }
221 }
222 }
223
224 @Test
225 public void testStandardParseParamsAndFillStreams() throws Exception
226 {
227 final String getParams = "qt=%C3%BC&dup=foo", postParams = "q=hello&d%75p=bar";
228 final byte[] postBytes = postParams.getBytes(StandardCharsets.US_ASCII);
229
230
231 final String[] ct = new String[] {
232 "application/x-www-form-urlencoded",
233 "Application/x-www-form-urlencoded",
234 "application/x-www-form-urlencoded; charset=utf-8",
235 "application/x-www-form-urlencoded;"
236 };
237
238 for( String contentType : ct ) {
239 HttpServletRequest request = getMock("/solr/select", contentType, postBytes.length);
240 expect(request.getMethod()).andReturn("POST").anyTimes();
241 expect(request.getQueryString()).andReturn(getParams).anyTimes();
242 expect(request.getInputStream()).andReturn(new ByteServletInputStream(postBytes));
243 replay(request);
244
245 MultipartRequestParser multipart = new MultipartRequestParser( 2048 );
246 RawRequestParser raw = new RawRequestParser();
247 FormDataRequestParser formdata = new FormDataRequestParser( 2048 );
248 StandardRequestParser standard = new StandardRequestParser( multipart, raw, formdata );
249
250 SolrParams p = standard.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
251
252 assertEquals( "contentType: "+contentType, "hello", p.get("q") );
253 assertEquals( "contentType: "+contentType, "\u00FC", p.get("qt") );
254 assertArrayEquals( "contentType: "+contentType, new String[]{"foo","bar"}, p.getParams("dup") );
255 }
256 }
257
258
259 static class ByteServletInputStream extends ServletInputStream {
260 final BufferedInputStream in;
261 final int len;
262 int readCount = 0;
263
264 public ByteServletInputStream(byte[] data) {
265 this.len = data.length;
266 this.in = new BufferedInputStream(new ByteArrayInputStream(data));
267 }
268
269 @Override
270 public boolean isFinished() {
271 return readCount == len;
272 }
273
274 @Override
275 public boolean isReady() {
276 return true;
277 }
278
279 @Override
280 public void setReadListener(ReadListener readListener) {
281 throw new IllegalStateException("Not supported");
282 }
283
284 @Override
285 public int read() throws IOException {
286 int read = in.read();
287 readCount += read;
288 return read;
289 }
290 }
291
292
293 @Test
294 public void testStandardParseParamsAndFillStreamsISO88591() throws Exception
295 {
296 final String getParams = "qt=%FC&dup=foo&ie=iso-8859-1&dup=%FC", postParams = "qt2=%FC&q=hello&d%75p=bar";
297 final byte[] postBytes = postParams.getBytes(StandardCharsets.US_ASCII);
298 final String contentType = "application/x-www-form-urlencoded; charset=iso-8859-1";
299
300
301 HttpServletRequest request = getMock("/solr/select", contentType, postBytes.length);
302 expect(request.getMethod()).andReturn("POST").anyTimes();
303 expect(request.getQueryString()).andReturn(getParams).anyTimes();
304 expect(request.getInputStream()).andReturn(new ByteServletInputStream(postBytes));
305 replay(request);
306
307 MultipartRequestParser multipart = new MultipartRequestParser( 2048 );
308 RawRequestParser raw = new RawRequestParser();
309 FormDataRequestParser formdata = new FormDataRequestParser( 2048 );
310 StandardRequestParser standard = new StandardRequestParser( multipart, raw, formdata );
311
312 SolrParams p = standard.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
313
314 assertEquals( "contentType: "+contentType, "hello", p.get("q") );
315 assertEquals( "contentType: "+contentType, "\u00FC", p.get("qt") );
316 assertEquals( "contentType: "+contentType, "\u00FC", p.get("qt2") );
317 assertArrayEquals( "contentType: "+contentType, new String[]{"foo","\u00FC","bar"}, p.getParams("dup") );
318 }
319
320 @Test
321 public void testStandardFormdataUploadLimit() throws Exception
322 {
323 final int limitKBytes = 128;
324
325 final StringBuilder large = new StringBuilder("q=hello");
326
327 while (large.length() <= limitKBytes * 1024) {
328 large.append('&').append(large);
329 }
330 HttpServletRequest request = getMock("/solr/select", "application/x-www-form-urlencoded", -1);
331 expect(request.getMethod()).andReturn("POST").anyTimes();
332 expect(request.getQueryString()).andReturn(null).anyTimes();
333 expect(request.getInputStream()).andReturn(new ByteServletInputStream(large.toString().getBytes(StandardCharsets.US_ASCII)));
334 replay(request);
335
336 FormDataRequestParser formdata = new FormDataRequestParser( limitKBytes );
337 try {
338 formdata.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
339 fail("should throw SolrException");
340 } catch (SolrException solre) {
341 assertTrue(solre.getMessage().contains("upload limit"));
342 assertEquals(400, solre.code());
343 }
344 }
345
346 @Test
347 public void testParameterIncompatibilityException1() throws Exception
348 {
349 HttpServletRequest request = getMock("/solr/select", "application/x-www-form-urlencoded", 100);
350 expect(request.getQueryString()).andReturn(null).anyTimes();
351
352 expect(request.getInputStream()).andReturn(new ServletInputStream() {
353 @Override
354 public int read() {
355 return -1;
356 }
357
358 @Override
359 public boolean isFinished() {
360 return true;
361 }
362
363 @Override
364 public boolean isReady() {
365 return true;
366 }
367
368 @Override
369 public void setReadListener(ReadListener readListener) {
370
371 }
372 });
373 replay(request);
374
375 FormDataRequestParser formdata = new FormDataRequestParser( 2048 );
376 try {
377 formdata.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
378 fail("should throw SolrException");
379 } catch (SolrException solre) {
380 assertTrue(solre.getMessage().startsWith("Solr requires that request parameters"));
381 assertEquals(500, solre.code());
382 }
383 }
384
385 @Test
386 public void testParameterIncompatibilityException2() throws Exception
387 {
388 HttpServletRequest request = getMock("/solr/select", "application/x-www-form-urlencoded", 100);
389 expect(request.getMethod()).andReturn("POST").anyTimes();
390 expect(request.getQueryString()).andReturn(null).anyTimes();
391
392 expect(request.getInputStream()).andThrow(new IllegalStateException());
393 replay(request);
394
395 FormDataRequestParser formdata = new FormDataRequestParser( 2048 );
396 try {
397 formdata.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
398 fail("should throw SolrException");
399 } catch (SolrException solre) {
400 assertTrue(solre.getMessage().startsWith("Solr requires that request parameters"));
401 assertEquals(500, solre.code());
402 }
403 }
404
405 @Test
406 public void testAddHttpRequestToContext() throws Exception {
407 HttpServletRequest request = getMock("/solr/select", null, -1);
408 expect(request.getMethod()).andReturn("GET").anyTimes();
409 expect(request.getQueryString()).andReturn("q=title:solr").anyTimes();
410 Map<String, String> headers = new HashMap<>();
411 headers.put("X-Forwarded-For", "10.0.0.1");
412 expect(request.getHeaderNames()).andReturn(new Vector<>(headers.keySet()).elements()).anyTimes();
413 for(Map.Entry<String,String> entry:headers.entrySet()) {
414 Vector<String> v = new Vector<>();
415 v.add(entry.getValue());
416 expect(request.getHeaders(entry.getKey())).andReturn(v.elements()).anyTimes();
417 }
418 replay(request);
419
420 SolrRequestParsers parsers = new SolrRequestParsers(h.getCore().getSolrConfig());
421 assertFalse(parsers.isAddRequestHeadersToContext());
422 SolrQueryRequest solrReq = parsers.parse(h.getCore(), "/select", request);
423 assertFalse(solrReq.getContext().containsKey("httpRequest"));
424
425 parsers.setAddRequestHeadersToContext(true);
426 solrReq = parsers.parse(h.getCore(), "/select", request);
427 assertEquals(request, solrReq.getContext().get("httpRequest"));
428 assertEquals("10.0.0.1", ((HttpServletRequest) solrReq.getContext().get("httpRequest")).getHeaders("X-Forwarded-For").nextElement());
429
430 }
431
432 public void testPostMissingContentType() throws Exception {
433 HttpServletRequest request = getMock();
434 expect(request.getMethod()).andReturn("POST").anyTimes();
435 expect(request.getQueryString()).andReturn(null).anyTimes();
436 expect(request.getHeader(anyObject(String.class))).andReturn(null).anyTimes();
437 replay(request);
438
439 SolrRequestParsers parsers = new SolrRequestParsers(h.getCore().getSolrConfig());
440 try {
441 parsers.parse(h.getCore(), "/select", request);
442 } catch (SolrException e) {
443 log.error("should not throw SolrException", e);
444 fail("should not throw SolrException");
445 }
446 }
447
448
449
450
451
452 @Test
453 public void testAutoDetect() throws Exception {
454 String curl = "curl/7.30.0";
455 for (String method : new String[]{"GET","POST"}) {
456 doAutoDetect(null, method, "{}=a", null, "{}", "a");
457 doAutoDetect(curl, method, "{}", "application/json", null, null);
458 doAutoDetect(curl, method, " \t\n\r {} ", "application/json", null, null);
459 doAutoDetect(curl, method, " \t\n\r // how now brown cow\n {} ", "application/json", null, null); // supporting comments
460 doAutoDetect(curl, method, " \t\n\r #different style comment\n {} ", "application/json", null, null);
461 doAutoDetect(curl, method, " \t\n\r /* C style comment */\n {} ", "application/json", null, null);
462 doAutoDetect(curl, method, " \t\n\r <tag>hi</tag> ", "text/xml", null, null);
463
464 doAutoDetect(curl, method, " \t\r\n aaa=1&bbb=2&ccc=3", null, "bbb", "2");
465 doAutoDetect(curl, method, "/x=foo&aaa=1&bbb=2&ccc=3", null, "/x", "foo");
466 doAutoDetect(curl, method, " \t\r\n /x=foo&aaa=1&bbb=2&ccc=3", null, "bbb", "2");
467 }
468 }
469
470 public void doAutoDetect(String userAgent, String method, final String body, String expectedContentType, String expectedKey, String expectedValue) throws Exception {
471 String uri = "/solr/select";
472 String contentType = "application/x-www-form-urlencoded";
473 int contentLength = -1;
474
475 HttpServletRequest request = createMock(HttpServletRequest.class);
476 expect(request.getHeader("User-Agent")).andReturn(userAgent).anyTimes();
477 expect(request.getHeader("Content-Length")).andReturn(null).anyTimes();
478 expect(request.getRequestURI()).andReturn(uri).anyTimes();
479 expect(request.getContentType()).andReturn(contentType).anyTimes();
480 expect(request.getContentLength()).andReturn(contentLength).anyTimes();
481 expect(request.getAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE)).andReturn(null).anyTimes();
482
483 expect(request.getMethod()).andReturn(method).anyTimes();
484
485 expect(request.getQueryString()).andReturn("foo=1&bar=2").anyTimes();
486 expect(request.getInputStream()).andReturn(new ByteServletInputStream(body.getBytes(StandardCharsets.US_ASCII)));
487 replay(request);
488
489 SolrRequestParsers parsers = new SolrRequestParsers(h.getCore().getSolrConfig());
490 SolrQueryRequest req = parsers.parse(h.getCore(), "/select", request);
491 int num=0;
492 if (expectedContentType != null) {
493 for (ContentStream cs : req.getContentStreams()) {
494 num++;
495 assertTrue(cs.getContentType().startsWith(expectedContentType));
496 String returnedBody = IOUtils.toString(cs.getReader());
497 assertEquals(body, returnedBody);
498 }
499 assertEquals(1, num);
500 }
501
502 assertEquals("1", req.getParams().get("foo"));
503 assertEquals("2", req.getParams().get("bar"));
504
505 if (expectedKey != null) {
506 assertEquals(expectedValue, req.getParams().get(expectedKey));
507 }
508
509 req.close();
510 }
511
512
513 public HttpServletRequest getMock() {
514 return getMock("/solr/select", null, -1);
515
516 }
517
518 public HttpServletRequest getMock(String uri, String contentType, int contentLength) {
519 HttpServletRequest request = createMock(HttpServletRequest.class);
520 expect(request.getHeader("User-Agent")).andReturn(null).anyTimes();
521 expect(request.getRequestURI()).andReturn(uri).anyTimes();
522 expect(request.getContentType()).andReturn(contentType).anyTimes();
523 expect(request.getContentLength()).andReturn(contentLength).anyTimes();
524 expect(request.getAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE)).andReturn(null).anyTimes();
525 return request;
526 }
527
528 }